//抓取安居客商业地产-写字楼经纪人列表
package main

import (
	//"bytes"
	"flag"
	"fmt"
	goq "lib/goquery"
	"lib/request"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	LOG         = ""
	NumCPU      = runtime.NumCPU()
	waitgroup   = &sync.WaitGroup{}
)

func init() {
	runtime.GOMAXPROCS(NumCPU)
	city_en := flag.String("city_en", "bj", "")
	flag.Parse()
	main_domain = "http://" + *city_en + ".xzl.anjuke.com/zu"
	LOG = *city_en + ".xzl.anjuke.com"
}

func main() {
	getDistrict(main_domain)
	s(time.Now().Format("2006-01-02 15:04:05"), "done.")
}

//读取区域列表
func getDistrict(url string) {
	doc, e := getDoc(url)
	if e != nil {
		errlog(e.Error())
		return
	}
	doc.Find(".fcklinks .cl_item").Eq(0).Find(".container a").Each(func(i int, sec *goq.Selection) {
		var (
			district string
			link     string
		)
		district = filterStr(sec.Text())
		link, ok := sec.Attr("href")
		if ok {
			getBlock(link, district)
		}
	})
}

//读取板块
func getBlock(url, district string) {
	doc, e := getDoc(url)
	if e != nil {
		errlog(e.Error())
		return
	}
	doc.Find(".fcklinks .cl_item").Eq(0).Find(".sub a").Each(func(i int, sec *goq.Selection) {
		var (
			block string
			link  string
		)
		block = filterStr(sec.Text())
		link, ok := sec.Attr("href")
		if ok {
			getHouseListPage(link, district, block)
		}
	})
}

//读取房源列表分页
func getHouseListPage(url string, district, block string) {
	doc, e := getDoc(url)
	if e != nil {
		errlog(e.Error())
		return
	}
	//recordCount
	recordCount, e := strconv.Atoi(filterStr(doc.Find(".spResult em").Text()))
	if e != nil {
		errlog(e.Error())
		return
	}
	pageSize := 25 //每页多少
	pageCount := math.Ceil(float64(recordCount) / float64(pageSize))
	var MaxPage float64 = 30 //只取前N页
	if pageCount < MaxPage {
		MaxPage = pageCount
	}
	var curpage float64 = 1
	for ; curpage <= MaxPage; curpage++ {
		tmpurl := fmt.Sprintf("%s-p%0.0f/#filtersort", url, curpage)
		tmpurl = strings.Replace(tmpurl, "/-", "-", -1)
		getHouseList(tmpurl, district, block)
	}
}

//读取房源列表
func getHouseList(url string, district, block string) {
	doc, e := getDoc(url)
	if e != nil {
		errlog(e.Error())
		return
	}

	type links struct {
		link, district, block string
	}
	linkContainer := []links{}
	doc.Find(".list li").Each(func(i int, sec *goq.Selection) {
		var (
			link string
		)
		link, ok := sec.Find(".details a").Attr("href")
		if ok {
			a := links{
				link:     link,
				district: district,
				block:    block,
			}
			linkContainer = append(linkContainer, a)
		}
	})

	for i := 0; i < len(linkContainer); i += NumCPU {
		for j := i; j < i+5 && j < len(linkContainer); j++ {
			waitgroup.Add(1)
			go getHouseDetail(linkContainer[j].link, linkContainer[j].district, linkContainer[j].block)
		}
		waitgroup.Wait()
	}
}

//读取房源最终页
func getHouseDetail(url string, district, block string) {
	defer waitgroup.Done()
	doc, e := getDoc(url)
	if e != nil {
		errlog(e.Error())
		return
	}
	var (
		broker_name  string //姓名
		broker_phone string //电话
		company      string //公司
		shop         string //店铺
	)
	broker_name = filterStr(doc.Find(".broker-name > a").Text())
	if broker_name == "" {
		broker_name = "--"
	}
	broker_phone = filterStr(doc.Find(".broker-phone .phone").Text())
	broker_phone = strings.Replace(broker_phone, string([]byte{194, 160}), "", -1)
	if broker_phone == "" {
		broker_phone = "--"
	}
	tmp := doc.Find(".company > .value")
	if tmp.Length() == 2 {
		company = filterStr(tmp.Eq(0).Text())
		shop = filterStr(tmp.Eq(1).Text())
	}
	if company == "" {
		company = "--"
	}
	if shop == "" {
		shop = "--"
	}
	outstr := ""
	outstr += district
	outstr += "#" + block
	outstr += "#" + company
	outstr += "#" + shop
	outstr += "#" + broker_name
	outstr += "#" + broker_phone
	outlog(outstr)
}

//读取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := request.Get(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "#", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//debuglog
func debuglog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_debug.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
